# -*- coding:utf-8 -*-

import requests
from lxml import etree
from fake_useragent import UserAgent
from http import cookiejar
import json
import time
from datetime import datetime, timedelta

requests.packages.urllib3.disable_warnings()  # 忽略HTTPS安全警告

"""
微博评论
https://m.weibo.cn/comments/hotflow?id=4451582411926548&mid=4451582411926548&max_id_type=0
"""

class WeiBo_Get():
    def __init__(self, since_date="2020-2-26"):
        #声明一个CookieJar对象实例来保存cookie
        self.cookie = cookiejar.CookieJar()
        # ua = UserAgent(use_cache_server=False)  # 禁用服务器缓存
        self.headers = {
            "cookie": "login_sid_t=1ccfda7d1051dc34b3ff66d4b1672716; cross_origin_proto=SSL; _s_tentry=passport.weibo.com; Apache=4847053334943.292.1624322017628; SINAGLOBAL=4847053334943.292.1624322017628; ULV=1624322017637:1:1:1:4847053334943.292.1624322017628:; ALF=1655857997; SSOLoginState=1624321997; SUB=_2A25N1V-eDeRhGeNN41sX8CjPyjWIHXVuozZWrDV8PUNbmtAKLWj8kW9NSZ0OGV-_KMw5s0HzECQi3CW9pN30GULD; SUBP=0033WrSXqPxfM725Ws9jqgMF55529P9D9WWyIgAfvDAPGCkEq6N8Kf4q5JpX5KzhUgL.Fo-01h.cehq0eK.2dJLoI0.LxK-LBo5L12qLxKML12eLB-2LxK-L1KeLB.eLxK-L1KeLB.eLxK-L1KBLBKMLxK-L1KBLBKMXeoqt; XSRF-TOKEN=gxloj2pwugjGGJnGJYCRP4X_; WBPSESS=UYmw2S8kYgl2SE4gm47Cxx8-1xmOfVfTagmbp0ivvOJYAH5Qx0sp8bt8xN-_CYMBtdmolkZfkcT5tGT94GbZEqu5Q3qn5srtC4QbPTiUfgeE0--ZAwra-T3PbEiGOvTP",
            "User-Agent": "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/79.0.3945.79 Safari/537.36",
        }
        # self.headers = {
        #     'authority': 'm.weibo.cn',
        #     'method': 'GET',
        #     'path': '/comments/hotFlowChild?cid=4528047065670664&max_id=4528077109207927&max_id_type=1',
        #     'scheme': 'https',
        #     'accept': 'application/json, text/plain, */*',
        #     'accept-encoding': 'gzip, deflate, br',
        #     'accept-language': 'zh-CN,zh;q=0.9',
        #     'cookie': '_T_WM=90042645475; WEIBOCN_FROM=1110006030; SUB=_2A25yfd63DeRhGeNN41sX8CjPyjWIHXVRgeL_rDV6PUJbktCOLWzHkW1NSZ0OGRXPi1Oj3mX-mxscgYaC0DbtP7jU; SUBP=0033WrSXqPxfM725Ws9jqgMF55529P9D9WWyIgAfvDAPGCkEq6N8Kf4q5NHD95Qfe0n4So5ce024Ws4Dqcj4i--fi-z7iKysi--NiKyhi-8Wi--fiK.0i-ihi--fiK.0i-ihi--fiK.Xi-2Ni--fiK.Xi-2NShzc; SUHB=0OMUchhUyGSQ0o; SSOLoginState=1601810152; MLOGIN=1; M_WEIBOCN_PARAMS=oid%3D4528046339526327%26luicode%3D20000061%26lfid%3D4528046339526327%26uicode%3D20000061%26fid%3D4528046339526327; XSRF-TOKEN=0dc982',
        #     'dnt': '1',
        #     'mweibo-pwa': '1',
        #     'referer': 'https://m.weibo.cn/detail/4528046339526327?cid=4528047065670664',
        #     'sec-fetch-dest': 'empty',
        #     'sec-fetch-mode': 'cors',
        #     'sec-fetch-site': 'same-origin',
        #     'user-agent': 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/85.0.4183.121 Safari/537.36',
        #     'x-requested-with': 'XMLHttpRequest',
        #     'x-xsrf-token': '0dc982'
        # }
        self.since_date = since_date

    # 获取用户信息
    def get_user(self, id):
        postData = {
            "containerid":"100505" + id, # 1669879400
        }

        url = "https://m.weibo.cn/api/container/getIndex"
        html = HttpUtils.do_request("GET", url, self.headers, postData)
        data_json = json.loads(html.text)
        info = data_json['data']['userInfo']
        user_info = {}
        # 根据containerid调用get_user_info获取用户的基本信息（所在地）
        user_info['containerid'] = data_json['data']['tabsInfo']['tabs'][0]['containerid']
        user_info['id'] = id  # 用户ID
        user_info['screen_name'] = info.get('screen_name', '')  # 昵称
        user_info['gender'] = info.get('gender', '')  # 性别
        user_info['statuses_count'] = info.get('statuses_count', 0)  # 微博数量
        user_info['followers_count'] = info.get('followers_count', 0)  # 粉丝数
        user_info['follow_count'] = info.get('follow_count', 0)  # 关注数
        user_info['description'] = info.get('description', '')  # 微博简介
        user_info['profile_url'] = info.get('profile_url', '')  # 微博主页
        user_info['profile_image_url'] = info.get('profile_image_url', '')  # 微博头像url
        user_info['avatar_hd'] = info.get('avatar_hd', '')  # 微博高清头像url
        user_info['urank'] = info.get('urank', 0)  # 等级
        user_info['mbrank'] = info.get('mbrank', 0)  # 会员等级
        user_info['verified'] = info.get('verified', False)  # 是否认证
        user_info['verified_type'] = info.get('verified_type', 0)  # 认证类型 没有认证值为-1，个人认证值为0，企业认证值为2，政府认证值为3
        user_info['verified_reason'] = info.get('verified_reason', '')  # 认证信息
        return user_info

    def print_weibo(self, weibo):
        """打印微博，若为转发微博，会同时打印原创和转发部分"""
        if weibo.get('retweet'):
            print("=============转发部分=============")
            self.print_one_weibo(weibo['retweet'])
            print("=============原创部分=============")
        self.print_one_weibo(weibo)
        print('=' * 120)


    def print_one_weibo(self, weibo):
        """打印一条微博"""
        print(u'微博id：%d' % weibo['id'])
        print(u'微博正文：%s' % weibo['text'])
        print(u'原始图片url：%s' % weibo['pics'])
        print(u'微博位置：%s' % weibo['location'])
        print(u'发布时间：%s' % weibo['created_at'])
        print(u'发布工具：%s' % weibo['source'])
        print(u'点赞数：%d' % weibo['attitudes_count'])
        print(u'评论数：%d' % weibo['comments_count'])
        print(u'转发数：%d' % weibo['reposts_count'])
        print(u'话题：%s' % weibo['topics'])
        print(u'@用户：%s' % weibo['at_users'])

    # 获取用户基本信息（所在地）
    def get_user_info(self, containerid):
        postData = {
            "containerid": containerid + "_-_INFO",#"2302831669879400_-_INFO",
            "title": "基本资料",
            "luicode": "10000011",
            "lfid": containerid#"2302831669879400",
        }

        url = "https://m.weibo.cn/api/container/getIndex"
        html = HttpUtils.do_request("GET", url, self.headers, postData)
        data_json = json.loads(html.text)
        user_info = {}
        user_info['sex'] = data_json['data']['cards'][1]['card_group'][1]['item_content'] # 性别
        user_info['address'] = data_json['data']['cards'][1]['card_group'][1]['item_content'] # 用户所在地



    # 获取微博发布内容
    def get_contents(self, id, page):
        postData = {
            "type": "uid",
            "page": page,  # "0",
            "containerid":"107603" + id, # 1669879400
        }
        res = True
        url = "https://m.weibo.cn/api/container/getIndex"
        html = HttpUtils.do_request("GET", url, self.headers, postData)
        data_json = json.loads(html.text)
        if data_json['ok']:
            weibos = data_json['data']['cards']
            for w in weibos:
                dict_data = dict()
                if w['card_type'] == 9:
                    weibo_info_json = w['mblog']  # 微博内容
                    weibo_id = weibo_info_json['id']
                    # 转发
                    if weibo_info_json.get('retweeted_status'):
                        retweet_id = weibo_info_json['retweeted_status']['id']  # 微博ID
                        is_long_retweet = weibo_info_json['retweeted_status']['isLongText']
                        if weibo_info_json['isLongText']:
                            dict_data = self.get_long_weibo(weibo_id)
                            if not dict_data:
                                dict_data = self.parse_weibo(weibo_info_json)
                        else:
                            #解析微博内容
                            dict_data = self.parse_weibo(weibo_info_json)
                        if is_long_retweet:
                            retweet = self.get_long_weibo(retweet_id)
                            if not retweet:
                                retweet = self.parse_weibo(weibo_info_json.get('retweeted_status'))
                        else:
                            retweet = self.parse_weibo(weibo_info_json.get('retweeted_status'))
                        dict_data['created_at'] = self.standardize_date(weibo_info_json['created_at'])
                        dict_data['retweet'] = retweet
                    # 原创
                    else:
                        if weibo_info_json['isLongText']:
                            dict_data = self.get_long_weibo(weibo_id)
                            if not dict_data:
                                dict_data = self.parse_weibo(weibo_info_json)
                        else:
                            #解析微博内容
                            dict_data = self.parse_weibo(weibo_info_json)
                    dict_data['created_at'] = self.standardize_date(weibo_info_json['created_at'])
                    created_at = datetime.strptime(
                        dict_data['created_at'], "%Y-%m-%d")
                    since_date = datetime.strptime(
                        self.since_date, "%Y-%m-%d")
                    if created_at < since_date:
                        print("================限定日期结束======================")
                        return None
                    # print(str(dict_data))
                    self.print_weibo(dict_data)
                    # 获取评论内容
                    # self.get_comments(dict_data['id'], 0)
            return res


    # 获取长微博
    def get_long_weibo(self, id):
        url = 'https://m.weibo.cn/detail/%s' % id
        html = HttpUtils.do_request("GET", url, "", "")
        html = html.text
        html = html[html.find('"status":'):]
        html = html[:html.rfind('"hotScheme"')]
        html = html[:html.rfind(',')]
        html = '{' + html + '}'
        js = json.loads(html, strict=False)
        weibo_info = js.get('status')
        if weibo_info:
            weibo = self.parse_weibo(weibo_info)
            return weibo


    # 解析微博信息:json-dict
    def parse_weibo(self, weibo_info_json):
        dict_data = dict()
        if weibo_info_json['user']:
            dict_data['user_id'] = weibo_info_json['user']['id']
            dict_data['screen_name'] = weibo_info_json['user']['screen_name']
        else:
            dict_data['user_id'] = ''
            dict_data['screen_name'] = ''
        dict_data['id'] = int(weibo_info_json['id'])
        dict_data['bid'] = weibo_info_json['bid']
        text_body = weibo_info_json['text']
        selector = etree.HTML(text_body)
        dict_data['text'] = etree.HTML(text_body).xpath('string(.)')  # 微博正文
        dict_data['pics'] = self.get_pics(weibo_info_json)  # 微博的原始图片url
        dict_data['video_url'] = self.get_video_url(weibo_info_json)  # 微博的视频url
        dict_data['location'] = self.get_location(selector)
        dict_data['created_at'] = weibo_info_json['created_at']  # 发布时间
        dict_data['source'] = weibo_info_json['source']  # 微博的发布工具
        dict_data['attitudes_count'] = self.string_to_int(
            weibo_info_json['attitudes_count'])  # 微博获得的点赞数
        dict_data['comments_count'] = self.string_to_int(
            weibo_info_json['comments_count'])  # 微博获得的评论数
        dict_data['reposts_count'] = self.string_to_int(
            weibo_info_json['reposts_count'])  # 微博获得的转发数
        dict_data['topics'] = self.get_topics(selector)  # 微博话题
        dict_data['at_users'] = self.get_at_users(selector)  # 微博@的用户
        return dict_data

    # 获取评论内容
    def get_comments(self, id, max_id, max_id_type):
        postData = {
            "id": id,  # "4456240593602073",  # 评论ID
            "mid": id,  # "4456240593602073",
            "max_id_type": "0",  # 默认0
        }
        # 第一次无max_id参数 游标
        if max_id != "0":
            postData['max_id'] = str(max_id)
            postData['max_id_type'] = str(max_id_type)
        url = "https://m.weibo.cn/comments/hotflow"
        html = HttpUtils.do_request("GET", url, self.headers, postData)
        data_json = json.loads(html.text)
        num = 0  # 计算已获取评论数量
        if data_json['ok'] == 1:
            count = data_json['data']['total_number']  # 总数量
            max_id = data_json['data']['max_id']  # 下页游标
            max_id_type = data_json['data']['max_id_type']  #
            for d in data_json['data']['data']:
                num += 1
                dict_data = dict()
                dict_data['id'] = d['id']  # 评论ID
                # 评论时间
                timeStruct = time.strptime(d['created_at'], '%a %b %d %X %z %Y')
                dict_data['created_at'] = time.strftime('%Y-%m-%d %H:%M:%S', timeStruct)
                dict_data['text'] = d['text']  # 评论内容
                dict_data['total_number'] = d['total_number']  # 回复数量
                dict_data['like_count'] = d['like_count']  # 点赞数量
                dict_data['user_id'] = d['user']['id']  # 用户id
                dict_data['user_name'] = d['user']['screen_name']  # 用户名称
                print(f"--------已获取评论数量：{num}-------------")
                print(str(dict_data))

                # 限定获取评论数量
                # if num >= 10:
                #     return
            time.sleep(1)
            self.get_comments(id, max_id, max_id_type)
        else:
            print("==========获取评论结束==================")


    # 获取评论回复内容
    def get_reply_comments(self, cid, max_id, max_id_type, count):
        postData = {
            "cid": cid,  # "4456240593602073",  # 评论ID
            "max_id": "0",  # "4456240593602073",
            "max_id_type": "0",  # 默认0
        }
        # 第一次无max_id参数 游标
        if max_id != "0":
            postData['max_id'] = str(max_id)
            postData['max_id_type'] = str(max_id_type)
        url = "https://m.weibo.cn/comments/hotFlowChild"
        html = HttpUtils.do_request("GET", url, self.headers, postData)
        data_json = json.loads(html.text)
        if data_json['ok'] == 1 and len(data_json['data']) > 0:
            total_count = data_json['total_number']  # 总数量
            max_id = data_json['max_id']  # 下页游标
            max_id_type = data_json['max_id_type']  #
            for d in data_json['data']:
                count += 1
                dict_data = dict()
                dict_data['id'] = d['id']  # 评论ID
                # 评论时间
                timeStruct = time.strptime(d['created_at'], '%a %b %d %X %z %Y')
                dict_data['created_at'] = time.strftime('%Y-%m-%d %H:%M:%S', timeStruct)
                dict_data['text'] = d['text']  # 评论内容
                dict_data['like_count'] = d['like_count']  # 点赞数量
                dict_data['user_id'] = d['user']['id']  # 用户id
                dict_data['user_name'] = d['user']['screen_name']  # 用户名称
                print(f"--------已获取回复数量：{count}-------------")
                print(str(dict_data))

                # 限定获取评论数量
                if count >= total_count:
                    return
            # if total_count > count:
            time.sleep(1)
            self.get_reply_comments(cid, max_id, max_id_type, count)
        else:
            print("==========获取评论结束==================")

    # 获取图片url
    def get_pics(self, weibo_info):
        if weibo_info.get('pics'):
            pic_info = weibo_info['pics']
            pic_list = [pic['large']['url'] for pic in pic_info]
            pics = ','.join(pic_list)
        else:
            pics = ''
        return pics


    # 获取视频url
    def get_video_url(self, weibo_info):
        video_url = ""
        if weibo_info.get('page_info'):
            if weibo_info['page_info'].get('media_info'):
                media_info = weibo_info['page_info']['media_info']
                video_url = media_info.get('mp4_720p_mp4')
                if not video_url:
                    video_url = media_info.get('mp4_hd_url')
                    if not video_url:
                        video_url = media_info.get('mp4_sd_url')
                        if not video_url:
                            video_url = ''
        return video_url


    # 获取发布位置
    def get_location(self, selector):
        location_icon = "timeline_card_small_location_default.png"
        span_list = selector.xpath('//span')
        location = ""
        for i, span in enumerate(span_list):
            if span.xpath('img/@src'):
                if location_icon in span.xpath('img/@src')[0]:
                    location = span_list[i + 1].xpath('string(.)')
                    break
        return location


    # 获取@用户
    def get_at_users(self, selector):
        a_list = selector.xpath('//a')
        at_users = ''
        at_list = []
        for a in a_list:
            if '@' + a.xpath('@href')[0][3:] == a.xpath('string(.)'):
                at_list.append(a.xpath('string(.)')[1:])
        if at_list:
            at_users = ",".join(at_list)
        return at_users


    # 获取参与的微博话题
    def get_topics(self, selector):
        span_list = selector.xpath('//span[@class="url-text"]')
        topics = ""
        topic_list = []
        for span in span_list:
            text = span.xpath('string(.)')
            if len(text) > 2 and text[0] == "#" and text[-1] == "#":
                topic_list.append(text[1:-1])
        if topic_list:
            topics = ",".join(topic_list)
        return topics


    # 转换时间
    def standardize_date(self, created_at):
        """标准化微博发布时间"""
        if u"刚刚" in created_at:
            created_at = datetime.now().strftime("%Y-%m-%d")
        elif u"分钟" in created_at:
            minute = created_at[:created_at.find(u"分钟")]
            minute = timedelta(minutes=int(minute))
            created_at = (datetime.now() - minute).strftime("%Y-%m-%d")
        elif u"小时" in created_at:
            hour = created_at[:created_at.find(u"小时")]
            hour = timedelta(hours=int(hour))
            created_at = (datetime.now() - hour).strftime("%Y-%m-%d")
        elif u"昨天" in created_at:
            day = timedelta(days=1)
            created_at = (datetime.now() - day).strftime("%Y-%m-%d")
        elif created_at.count('-') == 1:
            year = datetime.now().strftime("%Y")
            created_at = year + "-" + created_at
        return created_at


    # 字符串转换为整数
    def string_to_int(self, string):
        if isinstance(string, int):
            return string
        elif string.endswith(u'万+'):
            string = int(string[:-2] + '0000')
        elif string.endswith(u'万'):
            string = int(string[:-1] + '0000')
        return int(string)


if __name__ == '__main__':
    since_date = "2020-03-17"  # 微博截止时间
    weibo_get = WeiBo_Get(since_date)
    # id = "1669879400"
    # # id = "2656274875"
    # # 获取用户信息
    # weibo_user = weibo_get.get_user(id)
    # # 获取用户(所在地)
    # containerid = weibo_user['containerid']
    # # test_get.get_user_info(containerid)
    # for p in range(1, weibo_user['statuses_count']):
    #     # 获取发布微博内容
    #     result = weibo_get.get_contents(id, p)
    #     if result == None:
    #         break

    # 获取微博评论内容
    id = "4555315897444552"

    weibo_get.get_comments(id, "0", "0")

    # 获取回复内容
    # weibo_get.get_reply_comments("4528047065670664", "0", "0", 0)
    weibo_get.get_reply_comments("4527771864276259", "0", "0", 0)